latin1prober.py 5.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145
  1. ######################## BEGIN LICENSE BLOCK ########################
  2. # The Original Code is Mozilla Universal charset detector code.
  3. #
  4. # The Initial Developer of the Original Code is
  5. # Netscape Communications Corporation.
  6. # Portions created by the Initial Developer are Copyright (C) 2001
  7. # the Initial Developer. All Rights Reserved.
  8. #
  9. # Contributor(s):
  10. # Mark Pilgrim - port to Python
  11. # Shy Shalom - original C code
  12. #
  13. # This library is free software; you can redistribute it and/or
  14. # modify it under the terms of the GNU Lesser General Public
  15. # License as published by the Free Software Foundation; either
  16. # version 2.1 of the License, or (at your option) any later version.
  17. #
  18. # This library is distributed in the hope that it will be useful,
  19. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  20. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  21. # Lesser General Public License for more details.
  22. #
  23. # You should have received a copy of the GNU Lesser General Public
  24. # License along with this library; if not, write to the Free Software
  25. # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
  26. # 02110-1301 USA
  27. ######################### END LICENSE BLOCK #########################
  28. from .charsetprober import CharSetProber
  29. from .enums import ProbingState
  30. FREQ_CAT_NUM = 4
  31. UDF = 0 # undefined
  32. OTH = 1 # other
  33. ASC = 2 # ascii capital letter
  34. ASS = 3 # ascii small letter
  35. ACV = 4 # accent capital vowel
  36. ACO = 5 # accent capital other
  37. ASV = 6 # accent small vowel
  38. ASO = 7 # accent small other
  39. CLASS_NUM = 8 # total classes
  40. Latin1_CharToClass = (
  41. OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # 00 - 07
  42. OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # 08 - 0F
  43. OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # 10 - 17
  44. OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # 18 - 1F
  45. OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # 20 - 27
  46. OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # 28 - 2F
  47. OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # 30 - 37
  48. OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # 38 - 3F
  49. OTH, ASC, ASC, ASC, ASC, ASC, ASC, ASC, # 40 - 47
  50. ASC, ASC, ASC, ASC, ASC, ASC, ASC, ASC, # 48 - 4F
  51. ASC, ASC, ASC, ASC, ASC, ASC, ASC, ASC, # 50 - 57
  52. ASC, ASC, ASC, OTH, OTH, OTH, OTH, OTH, # 58 - 5F
  53. OTH, ASS, ASS, ASS, ASS, ASS, ASS, ASS, # 60 - 67
  54. ASS, ASS, ASS, ASS, ASS, ASS, ASS, ASS, # 68 - 6F
  55. ASS, ASS, ASS, ASS, ASS, ASS, ASS, ASS, # 70 - 77
  56. ASS, ASS, ASS, OTH, OTH, OTH, OTH, OTH, # 78 - 7F
  57. OTH, UDF, OTH, ASO, OTH, OTH, OTH, OTH, # 80 - 87
  58. OTH, OTH, ACO, OTH, ACO, UDF, ACO, UDF, # 88 - 8F
  59. UDF, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # 90 - 97
  60. OTH, OTH, ASO, OTH, ASO, UDF, ASO, ACO, # 98 - 9F
  61. OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # A0 - A7
  62. OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # A8 - AF
  63. OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # B0 - B7
  64. OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # B8 - BF
  65. ACV, ACV, ACV, ACV, ACV, ACV, ACO, ACO, # C0 - C7
  66. ACV, ACV, ACV, ACV, ACV, ACV, ACV, ACV, # C8 - CF
  67. ACO, ACO, ACV, ACV, ACV, ACV, ACV, OTH, # D0 - D7
  68. ACV, ACV, ACV, ACV, ACV, ACO, ACO, ACO, # D8 - DF
  69. ASV, ASV, ASV, ASV, ASV, ASV, ASO, ASO, # E0 - E7
  70. ASV, ASV, ASV, ASV, ASV, ASV, ASV, ASV, # E8 - EF
  71. ASO, ASO, ASV, ASV, ASV, ASV, ASV, OTH, # F0 - F7
  72. ASV, ASV, ASV, ASV, ASV, ASO, ASO, ASO, # F8 - FF
  73. )
  74. # 0 : illegal
  75. # 1 : very unlikely
  76. # 2 : normal
  77. # 3 : very likely
  78. Latin1ClassModel = (
  79. # UDF OTH ASC ASS ACV ACO ASV ASO
  80. 0, 0, 0, 0, 0, 0, 0, 0, # UDF
  81. 0, 3, 3, 3, 3, 3, 3, 3, # OTH
  82. 0, 3, 3, 3, 3, 3, 3, 3, # ASC
  83. 0, 3, 3, 3, 1, 1, 3, 3, # ASS
  84. 0, 3, 3, 3, 1, 2, 1, 2, # ACV
  85. 0, 3, 3, 3, 3, 3, 3, 3, # ACO
  86. 0, 3, 1, 3, 1, 1, 1, 3, # ASV
  87. 0, 3, 1, 3, 1, 1, 3, 3, # ASO
  88. )
  89. class Latin1Prober(CharSetProber):
  90. def __init__(self):
  91. super(Latin1Prober, self).__init__()
  92. self._last_char_class = None
  93. self._freq_counter = None
  94. self.reset()
  95. def reset(self):
  96. self._last_char_class = OTH
  97. self._freq_counter = [0] * FREQ_CAT_NUM
  98. CharSetProber.reset(self)
  99. @property
  100. def charset_name(self):
  101. return "ISO-8859-1"
  102. @property
  103. def language(self):
  104. return ""
  105. def feed(self, byte_str):
  106. byte_str = self.filter_with_english_letters(byte_str)
  107. for c in byte_str:
  108. char_class = Latin1_CharToClass[c]
  109. freq = Latin1ClassModel[(self._last_char_class * CLASS_NUM)
  110. + char_class]
  111. if freq == 0:
  112. self._state = ProbingState.NOT_ME
  113. break
  114. self._freq_counter[freq] += 1
  115. self._last_char_class = char_class
  116. return self.state
  117. def get_confidence(self):
  118. if self.state == ProbingState.NOT_ME:
  119. return 0.01
  120. total = sum(self._freq_counter)
  121. if total < 0.01:
  122. confidence = 0.0
  123. else:
  124. confidence = ((self._freq_counter[3] - self._freq_counter[1] * 20.0)
  125. / total)
  126. if confidence < 0.0:
  127. confidence = 0.0
  128. # lower the confidence of latin1 so that other more accurate
  129. # detector can take priority.
  130. confidence = confidence * 0.73
  131. return confidence